package com.xhji.spider.utils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;

public class SpiderWebUtil {

    public static Document getDoc(String url) {
        try {
            Document document = Jsoup.connect(url).timeout(10000).ignoreContentType(true)
                    .header("User-Agent",
                            "Mozilla/5.0 (Linux; U; Android 6.0.1; zh-CN; SM-C7000 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2214.89 UCBrowser/11.6.2.948 Mobile Safari/537.36")
                    .get();
            return document;
        } catch (Exception e) {
            System.out.println("url链接访问失败"+url);
            System.out.println("失败原因"+e.getMessage());
        }
        return null;
    }

    /**
     * 根据当前url获取该类别下分页条件下的所有url 第一页需要根据实际情况添加
     * 如：https://gitee.com/explore/all?lang=Java&order=starred
     * https://gitee.com/explore/all?lang=Java&order=starred&page=2
     * @param url https://gitee.com/explore/all?lang=Java&order=starred&page=%s
     * @param total 100
     * @return list
     */
    public static List<String> createCatalogUrls(String url, int total){
        List<String> list = new ArrayList<>(total);
        for (int i =1;i<=total;i++){
            list.add(String.format(url,i));
        }
        return list;
    }

    public static List<String> createPageUrl(String url, String className){
        try {
            Document document = getDoc(url);
            if (document == null)return null;
            Elements select = document.getElementsByClass(className);
            select.forEach(element -> {
                System.out.println(element.text());
            });
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }

    public static List<String> createImagesUrl(String url, String className){
        Document doc = getDoc(url);
        if (doc==null)return null;
        Elements select = doc.getElementsByClass(className);
        select.forEach(element -> {
            System.out.println(element.text());
        });
        return null;
    }

    public static void main(String[] args) {
        createPageUrl("https://gitee.com/explore/all?lang=Java&order=starred&page=1", "title project-namespace-path");
        createImagesUrl("https://gitee.com/shuzheng/zheng","file_content markdown-body");
    }
}
